package com.play.parse;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.Logger;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.parser.BinaryParseData;
import edu.uci.ics.crawler4j.parser.ExtractedUrlAnchorPair;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.parser.Parser;
import edu.uci.ics.crawler4j.parser.TextParseData;
import edu.uci.ics.crawler4j.url.URLCanonicalizer;
import edu.uci.ics.crawler4j.url.WebURL;
import edu.uci.ics.crawler4j.util.Util;

/**
 * 重写的Parser 
 * @author Administrator
 *
 */
public class MyParser extends  Parser{
	protected static final Logger logger = Logger.getLogger(MyParser.class.getName());
	public MyParser(CrawlConfig config) {
		super(config);
	}
	
	JsoupHandler jsoupHandler ;
	public MyParser(CrawlConfig config,JsoupHandler jsoupHandler) {
		super(config);
		this.jsoupHandler = jsoupHandler;
	}
	
	
	public boolean parse(Page page, String contextURL) {
		byte[] contentData = page.getContentData();
		if (Util.hasBinaryContent(page.getContentType())) {
			if (!config.isIncludeBinaryContentInCrawling()) {
				return false;
			}

			page.setParseData(BinaryParseData.getInstance());
			return true;

		} else if (Util.hasPlainTextContent(page.getContentType())) {
			try {
				TextParseData parseData = new TextParseData();
				if (page.getContentCharset() == null) {
					parseData.setTextContent(new String(contentData));
				} else {
					parseData.setTextContent(new String(contentData, page.getContentCharset()));
				}
				page.setParseData(parseData);
				return true;
			} catch (Exception e) {
				logger.error(e.getMessage() + ", while parsing: " + page.getWebURL().getURL());
			}
			return false;
		}

		//TODO kongxp这里可以改为配置方式
		
		InputStream inputStream = null;
		try {
			inputStream = new ByteArrayInputStream(contentData);
			//System.out.println(new String(contentData,"GBK"));
		
			jsoupHandler.parse(contentData,contextURL);
		} catch (Exception e) {
			e.printStackTrace();
			logger.error(e.getMessage() + ", while parsing: " + page.getWebURL().getURL());
		} finally {
			try {
				if (inputStream != null) {
					inputStream.close();
				}
			} catch (IOException e) {
				logger.error(e.getMessage() + ", while parsing: " + page.getWebURL().getURL());
			}
		}

		if (page.getContentCharset() == null) {
			page.setContentCharset(jsoupHandler.getContentCharset());
		}

		HtmlParseData parseData = new HtmlParseData();
		parseData.setText(jsoupHandler.getBodyText());
		parseData.setTitle(jsoupHandler.getTitle());

		List<WebURL> outgoingUrls = new ArrayList<>();

		String baseURL = jsoupHandler.getBaseUrl();
		if (baseURL != null) {
			contextURL = baseURL;
		}

		int urlCount = 0;
		for (ExtractedUrlAnchorPair urlAnchorPair : jsoupHandler.getOutgoingUrls()) {
			String href = urlAnchorPair.getHref();
			href = href.trim();
			if (href.length() == 0) {
				continue;
			}
			String hrefWithoutProtocol = href.toLowerCase();
			if (href.startsWith("http://")) {
				hrefWithoutProtocol = href.substring(7);
			}
			if (!hrefWithoutProtocol.contains("javascript:") && !hrefWithoutProtocol.contains("mailto:")
					&& !hrefWithoutProtocol.contains("@")) {
				String url = URLCanonicalizer.getCanonicalURL(href, contextURL);
				if (url != null) {
					WebURL webURL = new WebURL();
					webURL.setURL(url);
					webURL.setAnchor(urlAnchorPair.getAnchor());
					outgoingUrls.add(webURL);
					urlCount++;
					if (urlCount > config.getMaxOutgoingLinksToFollow()) {
						break;
					}
				}
			}
		}

		parseData.setOutgoingUrls(outgoingUrls);

		try {
			if (page.getContentCharset() == null) {
				parseData.setHtml(new String(contentData));
			} else {
				parseData.setHtml(new String(contentData, page.getContentCharset()));
			}
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
			return false;
		}

		page.setParseData(parseData);
		return true;
	}
}
